<!DOCTYPE html>
<html lang="zh-CN">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>强化学习RNN吃豆人</title>
    <style>
        body {
            margin: 0;
            padding: 20px;
            background: #000;
            color: #fff;
            font-family: Arial, sans-serif;
            display: flex;
            flex-direction: column;
            align-items: center;
        }
        
        #gameContainer {
            position: relative;
            border: 2px solid #00ff00;
            background: #000;
        }
        
        #gameCanvas {
            display: block;
            background: #000;
        }
        
        #neuralCanvas {
            display: block;
            background: #111;
            border: 1px solid #333;
            margin-top: 10px;
        }
        
        #scoreBoard {
            margin-top: 20px;
            text-align: center;
            font-size: 18px;
            display: grid;
            grid-template-columns: 1fr 1fr 1fr;
            gap: 20px;
        }
        
        #controls {
            margin-top: 20px;
            text-align: center;
        }
        
        button {
            padding: 10px 20px;
            margin: 0 5px;
            background: #00ff00;
            color: #000;
            border: none;
            cursor: pointer;
            font-size: 14px;
        }
        
        button:hover {
            background: #00cc00;
        }
        
        button:disabled {
            background: #666;
            cursor: not-allowed;
        }
        
        #trainingStats {
            margin-top: 20px;
            text-align: center;
            font-size: 14px;
        }
        
        .stat-item {
            margin: 5px 0;
            color: #0ff;
        }
        
        #modeDisplay {
            color: #ff0;
            font-weight: bold;
        }
        
        .progress-bar {
            width: 200px;
            height: 20px;
            background: #333;
            border: 1px solid #666;
            margin: 10px auto;
        }
        
        .progress-fill {
            height: 100%;
            background: #00ff00;
            transition: width 0.3s;
        }
    </style>
</head>
<body>
    <h1>强化学习RNN吃豆人</h1>
    
    <div id="gameContainer">
        <canvas id="gameCanvas" width="600" height="600"></canvas>
        <canvas id="neuralCanvas" width="600" height="200"></canvas>
    </div>
    
    <div id="scoreBoard">
        <div>
            <div>当前分数: <span id="currentScore">0</span></div>
            <div>最高分: <span id="highScore">0</span></div>
        </div>
        <div>
            <div>模式: <span id="modeDisplay">手动</span></div>
            <div>训练回合: <span id="episodeCount">0</span></div>
        </div>
        <div>
            <div>学习率: <span id="learningRate">0.001</span></div>
            <div>探索率: <span id="epsilon">1.0</span></div>
        </div>
    </div>
    
    <div id="controls">
        <button onclick="toggleMode()">切换模式</button>
        <button onclick="startTraining()" id="trainBtn">开始训练</button>
        <button onclick="stopTraining()" id="stopBtn" disabled>停止训练</button>
        <button onclick="resetGame()">重置游戏</button>
        <button onclick="saveModel()">保存模型</button>
        <button onclick="loadModel()">加载模型</button>
    </div>
    
    <div id="trainingStats">
        <div class="stat-item">平均奖励: <span id="avgReward">0</span></div>
        <div class="stat-item">训练进度:</div>
        <div class="progress-bar">
            <div class="progress-fill" id="progressBar" style="width: 0%"></div>
        </div>
    </div>
    
    <div id="instructions">
        <p>方向键/WASD: 手动控制 | 空格: 切换模式 | T: 训练模式</p>
        <p>强化学习RNN架构：状态→RNN→Q值→动作→奖励→学习</p>
    </div>

    <script>
        // 神经网络类
        class RNN {
            constructor(inputSize, hiddenSize, outputSize) {
                this.inputSize = inputSize;
                this.hiddenSize = hiddenSize;
                this.outputSize = outputSize;
                
                // 权重初始化
                this.Wxh = this.randomMatrix(inputSize, hiddenSize);
                this.Whh = this.randomMatrix(hiddenSize, hiddenSize);
                this.Why = this.randomMatrix(hiddenSize, outputSize);
                this.bh = new Array(hiddenSize).fill(0);
                this.by = new Array(outputSize).fill(0);
                
                // 隐藏状态
                this.h = new Array(hiddenSize).fill(0);
                
                // 存储训练数据
                this.memory = [];
                this.maxMemory = 10000;
            }
            
            randomMatrix(rows, cols) {
                const matrix = [];
                for (let i = 0; i < rows; i++) {
                    matrix[i] = [];
                    for (let j = 0; j < cols; j++) {
                        matrix[i][j] = (Math.random() - 0.5) * 0.1;
                    }
                }
                return matrix;
            }
            
            sigmoid(x) {
                return 1 / (1 + Math.exp(-x));
            }
            
            tanh(x) {
                return Math.tanh(x);
            }
            
            relu(x) {
                return Math.max(0, x);
            }
            
            forward(input) {
                // 输入层到隐藏层
                const h_raw = new Array(this.hiddenSize).fill(0);
                for (let i = 0; i < this.hiddenSize; i++) {
                    for (let j = 0; j < this.inputSize; j++) {
                        h_raw[i] += input[j] * this.Wxh[j][i];
                    }
                    for (let j = 0; j < this.hiddenSize; j++) {
                        h_raw[i] += this.h[j] * this.Whh[j][i];
                    }
                    h_raw[i] += this.bh[i];
                }
                
                // 更新隐藏状态
                this.h = h_raw.map(x => this.tanh(x));
                
                // 隐藏层到输出层
                const output = new Array(this.outputSize).fill(0);
                for (let i = 0; i < this.outputSize; i++) {
                    for (let j = 0; j < this.hiddenSize; j++) {
                        output[i] += this.h[j] * this.Why[j][i];
                    }
                    output[i] += this.by[i];
                }
                
                return output;
            }
            
            train(state, action, reward, nextState, learningRate = 0.001) {
                // 经验回放
                this.memory.push({state, action, reward, nextState});
                if (this.memory.length > this.maxMemory) {
                    this.memory.shift();
                }
                
                if (this.memory.length < 32) return; // 等待足够经验
                
                // 随机采样小批量
                const batch = [];
                for (let i = 0; i < 32; i++) {
                    const idx = Math.floor(Math.random() * this.memory.length);
                    batch.push(this.memory[idx]);
                }
                
                // 简单的Q学习更新
                for (const exp of batch) {
                    const qValues = this.forward(exp.state);
                    const nextQValues = this.forward(exp.nextState);
                    
                    const target = exp.reward + 0.99 * Math.max(...nextQValues);
                    const error = target - qValues[exp.action];
                    
                    // 更新权重（简化版）
                    for (let i = 0; i < this.outputSize; i++) {
                        if (i === exp.action) {
                            this.by[i] += learningRate * error;
                        }
                    }
                }
            }
            
            resetHiddenState() {
                this.h = new Array(this.hiddenSize).fill(0);
            }
        }
        
        // 游戏配置
        const CELL_SIZE = 20;
        const GRID_WIDTH = 30;
        const GRID_HEIGHT = 30;
        const ACTIONS = ['up', 'down', 'left', 'right', 'stay'];
        
        // 游戏状态
        let gameState = {
            score: 0,
            highScore: parseInt(localStorage.getItem('rlHighScore') || '0'),
            mode: 'manual', // manual, ai, training
            pacman: { x: 1, y: 1 },
            ghosts: [],
            pellets: [],
            walls: [],
            episode: 0,
            totalReward: 0,
            epsilon: 1.0,
            epsilonDecay: 0.995,
            minEpsilon: 0.01
        };
        
        // 神经网络
        let rnn = new RNN(10, 64, 5); // 10个状态特征，64个隐藏单元，5个动作
        
        // 画布
        const gameCanvas = document.getElementById('gameCanvas');
        const gameCtx = gameCanvas.getContext('2d');
        const neuralCanvas = document.getElementById('neuralCanvas');
        const neuralCtx = neuralCanvas.getContext('2d');
        
        // 初始化地图
        function initMap() {
            gameState.walls = [];
            gameState.pellets = [];
            
            // 边界墙
            for (let x = 0; x < GRID_WIDTH; x++) {
                gameState.walls.push({x, y: 0});
                gameState.walls.push({x, y: GRID_HEIGHT - 1});
            }
            for (let y = 0; y < GRID_HEIGHT; y++) {
                gameState.walls.push({x: 0, y});
                gameState.walls.push({x: GRID_WIDTH - 1, y});
            }
            
            // 内部障碍物
            const obstacles = [
                [5, 5, 10, 1], [15, 5, 1, 10], [20, 10, 10, 1],
                [5, 15, 10, 1], [10, 20, 1, 8], [20, 20, 8, 1]
            ];
            
            obstacles.forEach(([x, y, w, h]) => {
                for (let dx = 0; dx < w; dx++) {
                    for (let dy = 0; dy < h; dy++) {
                        gameState.walls.push({x: x + dx, y: y + dy});
                    }
                }
            });
            
            // 生成豆子
            for (let x = 1; x < GRID_WIDTH - 1; x++) {
                for (let y = 1; y < GRID_HEIGHT - 1; y++) {
                    if (!isWall(x, y)) {
                        gameState.pellets.push({x, y});
                    }
                }
            }
        }
        
        function isWall(x, y) {
            return gameState.walls.some(wall => wall.x === x && wall.y === y);
        }
        
        function initGhosts() {
            gameState.ghosts = [
                {x: 15, y: 15, color: 'red', dx: 1, dy: 0},
                {x: 20, y: 20, color: 'pink', dx: -1, dy: 0},
                {x: 10, y: 20, color: 'cyan', dx: 0, dy: 1}
            ];
        }
        
        // 获取游戏状态特征
        function getStateFeatures() {
            const pacman = gameState.pacman;
            const features = [];
            
            // 1-4: 四个方向的墙壁检测
            features.push(isWall(pacman.x + 1, pacman.y) ? 1 : 0);
            features.push(isWall(pacman.x - 1, pacman.y) ? 1 : 0);
            features.push(isWall(pacman.x, pacman.y + 1) ? 1 : 0);
            features.push(isWall(pacman.x, pacman.y - 1) ? 1 : 0);
            
            // 5-6: 最近的豆子方向
            let nearestPellet = findNearestPellet();
            if (nearestPellet) {
                features.push((nearestPellet.x - pacman.x) / GRID_WIDTH);
                features.push((nearestPellet.y - pacman.y) / GRID_HEIGHT);
            } else {
                features.push(0, 0);
            }
            
            // 7-8: 最近的幽灵距离和方向
            let nearestGhost = findNearestGhost();
            if (nearestGhost) {
                const dist = Math.abs(nearestGhost.x - pacman.x) + Math.abs(nearestGhost.y - pacman.y);
                features.push(Math.min(1, 5 / dist));
                features.push(nearestGhost.x > pacman.x ? 1 : -1);
            } else {
                features.push(0, 0);
            }
            
            // 9-10: 豆子数量和游戏进度
            features.push(gameState.pellets.length / (GRID_WIDTH * GRID_HEIGHT));
            features.push(gameState.score / 1000);
            
            return features;
        }
        
        function findNearestPellet() {
            let nearest = null;
            let minDist = Infinity;
            
            for (let pellet of gameState.pellets) {
                const dist = Math.abs(pellet.x - gameState.pacman.x) + 
                           Math.abs(pellet.y - gameState.pacman.y);
                if (dist < minDist) {
                    minDist = dist;
                    nearest = pellet;
                }
            }
            
            return nearest;
        }
        
        function findNearestGhost() {
            let nearest = null;
            let minDist = Infinity;
            
            for (let ghost of gameState.ghosts) {
                const dist = Math.abs(ghost.x - gameState.pacman.x) + 
                           Math.abs(ghost.y - gameState.pacman.y);
                if (dist < minDist) {
                    minDist = dist;
                    nearest = ghost;
                }
            }
            
            return nearest;
        }
        
        // AI决策
        function getAIAction() {
            const state = getStateFeatures();
            const qValues = rnn.forward(state);
            
            // ε-贪婪策略
            if (Math.random() < gameState.epsilon) {
                return Math.floor(Math.random() * ACTIONS.length);
            }
            
            return qValues.indexOf(Math.max(...qValues));
        }
        
        // 执行动作
        function executeAction(action) {
            const moves = [
                {dx: 0, dy: -1},  // up
                {dx: 0, dy: 1},   // down
                {dx: -1, dy: 0},  // left
                {dx: 1, dy: 0},   // right
                {dx: 0, dy: 0}    // stay
            ];
            
            const move = moves[action];
            const newX = gameState.pacman.x + move.dx;
            const newY = gameState.pacman.y + move.dy;
            
            if (!isWall(newX, newY)) {
                gameState.pacman.x = newX;
                gameState.pacman.y = newY;
                
                // 检查吃豆子
                const pelletIndex = gameState.pellets.findIndex(
                    p => p.x === newX && p.y === newY
                );
                
                if (pelletIndex !== -1) {
                    gameState.pellets.splice(pelletIndex, 1);
                    gameState.score += 10;
                    return 10; // 吃豆子奖励
                }
            }
            
            return -0.1; // 小惩罚鼓励移动
        }
        
        // 移动幽灵
        function moveGhosts() {
            gameState.ghosts.forEach(ghost => {
                const directions = [
                    {dx: 1, dy: 0}, {dx: -1, dy: 0},
                    {dx: 0, dy: 1}, {dx: 0, dy: -1}
                ];
                
                const validMoves = directions.filter(dir => {
                    const newX = ghost.x + dir.dx;
                    const newY = ghost.y + dir.dy;
                    return !isWall(newX, newY);
                });
                
                if (validMoves.length > 0) {
                    const move = validMoves[Math.floor(Math.random() * validMoves.length)];
                    ghost.x += move.dx;
                    ghost.y += move.dy;
                }
                
                // 检查碰撞
                if (ghost.x === gameState.pacman.x && ghost.y === gameState.pacman.y) {
                    return -100; // 被抓住的大惩罚
                }
            });
            
            return 0;
        }
        
        // 绘制游戏
        function draw() {
            // 清空画布
            gameCtx.clearRect(0, 0, gameCanvas.width, gameCanvas.height);
            
            // 绘制墙
            gameCtx.fillStyle = '#0000ff';
            gameState.walls.forEach(wall => {
                gameCtx.fillRect(wall.x * CELL_SIZE, wall.y * CELL_SIZE, CELL_SIZE, CELL_SIZE);
            });
            
            // 绘制豆子
            gameCtx.fillStyle = '#ffff00';
            gameState.pellets.forEach(pellet => {
                gameCtx.beginPath();
                gameCtx.arc(
                    pellet.x * CELL_SIZE + CELL_SIZE/2,
                    pellet.y * CELL_SIZE + CELL_SIZE/2,
                    3, 0, Math.PI * 2
                );
                gameCtx.fill();
            });
            
            // 绘制吃豆人
            gameCtx.fillStyle = '#ffff00';
            gameCtx.beginPath();
            gameCtx.arc(
                gameState.pacman.x * CELL_SIZE + CELL_SIZE/2,
                gameState.pacman.y * CELL_SIZE + CELL_SIZE/2,
                CELL_SIZE/2 - 2, 0, Math.PI * 2
            );
            gameCtx.fill();
            
            // 绘制幽灵
            gameState.ghosts.forEach(ghost => {
                gameCtx.fillStyle = ghost.color;
                gameCtx.beginPath();
                gameCtx.arc(
                    ghost.x * CELL_SIZE + CELL_SIZE/2,
                    ghost.y * CELL_SIZE + CELL_SIZE/2,
                    CELL_SIZE/2 - 2, 0, Math.PI * 2
                );
                gameCtx.fill();
            });
            
            // 绘制神经网络可视化
            drawNeuralNetwork();
        }
        
        // 绘制神经网络
        function drawNeuralNetwork() {
            neuralCtx.clearRect(0, 0, neuralCanvas.width, neuralCanvas.height);
            
            // 绘制网络结构
            const state = getStateFeatures();
            const qValues = rnn.forward(state);
            
            // 输入层
            neuralCtx.fillStyle = '#0ff';
            for (let i = 0; i < 10; i++) {
                const x = 50 + i * 50;
                const y = 50;
                const size = Math.abs(state[i]) * 20 + 5;
                neuralCtx.beginPath();
                neuralCtx.arc(x, y, size, 0, Math.PI * 2);
                neuralCtx.fill();
            }
            
            // 隐藏层
            neuralCtx.fillStyle = '#0f0';
            for (let i = 0; i < 8; i++) {
                const x = 100 + i * 50;
                const y = 100;
                neuralCtx.beginPath();
                neuralCtx.arc(x, y, 8, 0, Math.PI * 2);
                neuralCtx.fill();
            }
            
            // 输出层（Q值）
            neuralCtx.fillStyle = '#ff0';
            const actions = ['↑', '↓', '←', '→', '○'];
            for (let i = 0; i < 5; i++) {
                const x = 100 + i * 100;
                const y = 150;
                const intensity = Math.max(0, qValues[i] / 10);
                neuralCtx.fillStyle = `rgba(255, 255, 0, ${intensity})`;
                neuralCtx.beginPath();
                neuralCtx.arc(x, y, 15, 0, Math.PI * 2);
                neuralCtx.fill();
                
                neuralCtx.fillStyle = '#fff';
                neuralCtx.font = '12px Arial';
                neuralCtx.textAlign = 'center';
                neuralCtx.fillText(actions[i], x, y + 4);
            }
            
            // 标签
            neuralCtx.fillStyle = '#fff';
            neuralCtx.font = '12px Arial';
            neuralCtx.textAlign = 'left';
            neuralCtx.fillText('输入层', 10, 55);
            neuralCtx.fillText('隐藏层', 10, 105);
            neuralCtx.fillText('Q值输出', 10, 155);
        }
        
        // 训练循环
        let trainingInterval = null;
        let episodeRewards = [];
        
        function startTraining() {
            gameState.mode = 'training';
            document.getElementById('trainBtn').disabled = true;
            document.getElementById('stopBtn').disabled = false;
            document.getElementById('modeDisplay').textContent =